package com.hefei

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

/**
 * @author caizhengjun
 * @date 2020/8/27 4:56 下午
 */
object WordCount {
  def main(args: Array[String]): Unit = {

    val conf = new SparkConf().setMaster("local[2]").setAppName(this.getClass.getSimpleName)
    val sc: SparkContext = new SparkContext(conf)

    //val source: RDD[String] = sc.textFile("dataset/wordcount.txt")
    val source: RDD[String] = sc.textFile("hdfs://work01:9000/user/caizhengjun/input")
    println(source.dependencies)

    val wordRDD: RDD[String] = source.flatMap(line => line.split(" "))
    println(wordRDD.dependencies)

    val wordsTupleRDD: RDD[(String, Int)] = wordRDD.map((_, 1))
    println(wordsTupleRDD.dependencies)

    val resultRDD: RDD[(String, Int)] = wordsTupleRDD.reduceByKey(_ + _)
    println(resultRDD.dependencies)

    println(resultRDD.collect().mkString(","))

    sc.stop()
  }
}
